package test;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.File;
import java.io.FileWriter;
import java.io.Writer;
import java.util.HashSet;
import java.util.Set;

public class AnalysisShopArea {
    public static void main(String[] args) throws Exception {
        Set<String> set = new HashSet<>();
        File[] files = new File("D://shopArea").listFiles();
        for (File file : files) {
            Document jsoup = Jsoup.parse(file, "utf-8");
            Element element = jsoup.getElementById("shop-all-list");
            Elements elements = element.select("a");
            for (Element e : elements) {
                String shop = e.attr("href");
                if (shop.contains("shop") && !shop.contains("review") && !shop.contains("#")) {
                    set.add(shop);
                }
            }
        }
        File file = new File("D:\\ideaCode\\Crawler\\src\\main\\java\\test\\shop.txt");
        Writer writer = new FileWriter(file);
        for (String s : set) {
            writer.write(s + "\n");
        }
        writer.flush();
        writer.close();
    }
}
